import scrapy

from ..items import MyspiderItem

class MianfeiSpider(scrapy.Spider):
    name = 'mianfei'
    allowed_domains = ['qidian.com']
    start_urls = ['http://www.qidian.com/free/all/action1/']

    def parse(self, response):
        # 解析文本
        sub_list = response.xpath('.//ul[@class="all-img-list cf"]/li')
        print(sub_list)
        # 遍历
        for i in sub_list :
            item = MyspiderItem()
            title = i.xpath('.//div[@class="book-mid-info"]/h2/a/text()').extract_first()
            item['title'] = title
            if title=="八荒剑神":
                continue
            url =i.xpath('.//h2/a/@href').extract_first()
            tmp = "https:"+url+"#Catalog"
            item['url'] = tmp
            yield scrapy.Request(url=tmp,callback=self.page_second,meta={'item':item})


    def page_second(self,response):
        '''
        中间页面
        :param response:
        :return:
        '''
        item = response.meta['item']

        # print("item-url--->",item['url'])
        # print("item--title--->",item['title'])
        wz_list= response.xpath('.//ul[@class="cf"]/li')

        for li in wz_list:
            # item['wzbt'] = li.xpath('//*[@id="j-catalogWrap"]/div[2]/div/ul/li/h2/a/text()').extract_first()
            item['wzbt'] = li.xpath('./h2/a/text()').extract_first()
            # print("文章章节名称:",item['wzbt'])
            # xx = li.xpath('./h2/a/@href')[0] //*[@id="j-catalogWrap"]/div[2]/div/ul/li[2]/h2/a
            hf = li.xpath('./h2/a/@href').extract_first()
            item['zj_url'] ="https:"+hf
            print("item:", item)
            # print("next--url_content:",url_content)
            yield scrapy.Request(url=item['zj_url'],callback=self.detail_content,meta={'item':item})


    def detail_content(self,response):
        item = response.meta['item']
        con =response.xpath('.//div[@class="read-content j_readContent"]//text()').extract()
        # print("con--->", con)
        cont = ''.join(con)
        item['content'] = cont.strip()
        yield item






